'''setycyas自制的下载器.
  读取同一文件夹下的links.txt文本,把每一行作为一个下载链接,#开头的作为注释跳过,
  同一文件夹下的succeeded.txt文本则记录所有成功下载的链接,
  failed.txt记录下载失败的链接.
  
  启动时请自行修改DOWNLOAD_PATH,指定下载的文件夹.
  启动后会遍历links.txt的所有下载链接,除了已经记录在succeeded.txt的以外.
  而failed.txt则只是给下载者自己看的,看看上次有哪些下载是失败的.
  '''
  
import sys
import os
import io
from threading import Thread, Lock
import urllib.request
import ssl

## 修改这里指定下载目录
DOWNLOAD_PATH = r'F://download/PyDownloadTest'
## 保证https可以使用
ssl._create_default_https_context = ssl._create_unverified_context

## 下载器类,用下载目录,下载线程数(默认5),下载链接文件,下载成功记录文件,
## 下载失败记录文件,指定的headers(默认为空)进行初始化,并使用main()方法开始下载.
class SetycyasDownloader:

  def __init__(self, downloadPath, threadsNumber = 5, 
    linksFile = sys.path[0]+'\\'+'links.txt',
    succeededFile = sys.path[0]+'\\'+'succeeded.txt',
    failedFile = sys.path[0]+'\\'+'failed.txt',
    headers = {}):
    ## 参数记录
    self.downloadPath = downloadPath
    self.threadsNumber = int(threadsNumber)
    self.linksFile = linksFile
    self.succeededFile = succeededFile
    self.failedFile = failedFile
    self.headers = headers
    ## 参数检查,初始化使用资源
    self.startErrorMsg = '' # 启动错误提示字符串,该字符串不为空时才能启动
    ## 启动参数检查
    if (not (os.path.isdir(self.downloadPath))):
      self.startErrorMsg += '错误!指定的下载目录不存在\n'
    if (not (os.path.exists(self.linksFile))):
      self.startErrorMsg += '错误!指定的下载列表文件不存在\n'
    if (self.threadsNumber <= 0):
      self.startErrorMsg += '错误!指定的线程数不是正整数\n'
    ## 检查记录文件,不存在时制作空文件.
    if (not (os.path.exists(self.linksFile))):
      with open(self.succeededFile, 'w') as f:
        pass
    if (not (os.path.exists(self.linksFile))):
      with open(self.failedFile, 'w') as f:
        pass
    ## 需要使用的资料准备
    self.linksList = [] # 下载链接表
    self.succeededList = [] # 成功表
    self.failedList = [] # 失败表
    self.lock = Lock() # 线程锁
    with open(self.succeededFile, 'r') as f:
      for line in f:
        _line = line.strip().replace('\n', '')
        if _line:
          self.succeededList.append(_line)
    with open(self.linksFile, 'r') as f:
      for line in f:
        _line = line.strip().replace('\n', '')
        if _line.startswith('#'):
          continue
        if (_line and (not (_line in self.succeededList))):
          self.linksList.append(_line)
    self.linksList.reverse()
    ## 准备完毕
    return    
      
  ## 下载一个链接,成功则写入文件并返回True,失败返回False
  def _downloadALink(self, link):
    # 这里要注意，必须使用url=url，headers=headers的格式，
    # 否则回报错，无法连接字符
    req = urllib.request.Request(url = link,headers = self.headers)   
    try:
      # 注意，这里要用req，不然就被添加useragent
      resp = urllib.request.urlopen(req)
      if(resp.status != 200):
        print('访问{0}失败,返回代码{1}'.format(link,resp.status))
        return False   
      else:
        filename = link.split('/')[-1].strip()
        filepath = self.downloadPath+'\\'+filename
        with open(filepath, 'wb') as f:
          f.write(resp.read())
        return True
    except Exception as e:
      print('访问{0}失败'.format(link),e)
      return False
    return
  
  ## 链接过滤,有效链接返回True
  def _linkFilter(self, link):
    if (not ('.' in link.split('/')[-1])):
      return False
    return True
  
  ## 下载待下载表中的下一个链接,直到下载列表为空,是一个单独的下载线程,
  ## id表示线程的名称,用于在显示时识别
  def _downloadThread(self, id):
    while(len(self.linksList) > 0):
      with self.lock:
        link = self.linksList.pop()
      if (not (link.startswith('http'))):
        link = 'http://'+link
      if (not (self._linkFilter(link))):
        continue
      downResult = self._downloadALink(link)
      with self.lock:
        if downResult:
          self.succeededList.append(link)
          print('线程{0}下载链接{1}成功!\n'.format(id, link))
        else:
          self.failedList.append(link)
          print('线程{0}下载链接{1}失败!\n'.format(id, link))
    return
      
  ## 启动用的主函数
  def main(self):
    if self.startErrorMsg:
      print(self.startErrorMsg)
      return
    threadsList=[] # 线程池
    for i in range(1, self.threadsNumber+1):
      t = Thread(target = self._downloadThread,args=(str(i),))
      threadsList.append(t)
      t.start()
    for t in threadsList:
      t.join()
    # 下载线程结束时:
    with open(self.succeededFile, 'w') as f:
      f.write('\n'.join(self.succeededList))
    with open(self.failedFile, 'w') as f:
      f.write('\n'.join(self.failedList))
    input('下载全部完成!成功{},失败{}.\n按回车结束'.format(
      len(self.succeededList), len(self.failedList)))

    
if __name__ == '__main__':
  
  headers =  {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36'
  }
  
  sd = SetycyasDownloader(DOWNLOAD_PATH, threadsNumber = 5, headers = headers)
  sd.main()